# -*- coding: utf-8 -*-
# @Time    : 2020/9/2 14:42
# @Author  : ZSQ
# @Email   : zsq199170918@163.com
# @FileName: proxy_to_redis_yueyang.py
# @Software: PyCharm
import time
import random
import requests
import threading
from loguru import logger
from selenium import webdriver
from redis import ConnectionPool, StrictRedis
from concurrent.futures import ThreadPoolExecutor
from selenium.common.exceptions import UnexpectedAlertPresentException

try:
    from FDC_spider.utils.redis_pool import POOL
except:
    HOST = '192.168.5.236'  # IP地址
    PORT = 6379  # 端口号
    DB = 4  # redis库号
    MAX_CONN = 50  # redis连接池最大连接数
    POOL = ConnectionPool(host=HOST, port=PORT, db=DB, max_connections=MAX_CONN, decode_responses=True)

PROXY_URL = [
    'http://192.168.5.223:5555/random',
    'http://192.168.5.233:5555/random',
]  # 代理url
PROXY_NUM = 10  # redis库中代理的数量
PROXY_YY_KEY = 'yy_proxy'  # redis库中代理的key名
TEST_URL = 'http://yyfdcw.com/NewHouse/BuildingList.aspx'  # 访问的url
TIME_OUT = 3  # 超时时间

CONN = StrictRedis(connection_pool=POOL)
PIPELINE = CONN.pipeline(transaction=True)


def get_proxy_ip() -> str:
    """
    获取代理IP  例如:'1.199.187.144:15087'
    :return: 代理IP
    """
    # 关闭长连接和多余连接
    headers = dict(Connection='close')
    _session = requests.session()
    _session.keep_alive = False

    resp = requests.get(random.choice(PROXY_URL), headers=headers)
    proxy_ip = resp.content.decode()
    del resp
    return proxy_ip


def set_script(host: str, port: str) -> str:
    """
    构造js 注入后更换代理
    :param host: IP
    :param port: 端口
    :return:
    """
    # js代码
    script_str = f"""
    var prefs = Components.classes["@mozilla.org/preferences-service;1"].getService(Components.interfaces.nsIPrefBranch);
    prefs.setIntPref("network.proxy.type", 1);
    prefs.setCharPref("network.proxy.http", "{host}");
    prefs.setIntPref("network.proxy.http_port", "{port}");
    prefs.setCharPref("network.proxy.ssl", "{host}");
    prefs.setIntPref("network.proxy.ssl_port", "{port}");
    prefs.setCharPref("network.proxy.ftp", "${host}");
    prefs.setIntPref("network.proxy.ftp_port", "{port}");
    """
    return script_str


def get_redis_key_len():
    """
    获取redis中代理池长度
    :return:
    """
    return CONN.llen(PROXY_YY_KEY)


def delete_redis_key(cycle=60 * 3):
    """
    清空redis中代理池
    :param cycle: 间隔时间 默认180s
    :return:
    """
    while True:
        # 默认每隔3分钟清空一次代理池，保证代理ip没有冗余、实时可用
        time.sleep(cycle)
        CONN.delete(PROXY_YY_KEY)


def check_proxy_ip(driver, proxy_ip) -> bool:
    """
    检查代理IP，并进行网页访问
    :param driver: 浏览器对象
    :param proxy_ip: 代理IP
    :return:
    """
    try:
        # 获取host、port
        host = proxy_ip.split(':')[0]
        port = proxy_ip.split(':')[1]
        # 设置超时
        driver.set_page_load_timeout(TIME_OUT)
        driver.set_script_timeout(TIME_OUT)
        # 访问火狐设置页
        driver.get("about:config")
        # 获取js
        script_str = set_script(host, port)
        # 添加代理
        driver.execute_script(script_str)
        # 访问网页
        driver.get(TEST_URL)
        time.sleep(4)
        # 判断是否被反爬
        assert 'equiv="refresh"' not in driver.page_source
    except UnexpectedAlertPresentException:
        driver.quit()
        error_msg = 'Unexpected Alert Present Exception'
        logger.error('{tid}: {e}', tid=threading.current_thread().name, e=error_msg)
        raise UnexpectedAlertPresentException
    except Exception as e:
        # logger.error('{tid}: {e}', tid=threading.current_thread().name, e=e)
        return False
    else:
        logger.info('{tid}: {ip}', tid=threading.current_thread().name, ip=proxy_ip)
        return True


def save_to_redis(proxy_ip) -> bool:
    """
    保存代理IP到redis
    :param proxy_ip: 代理IP
    :return:
    """
    try:
        # 开启redis事务
        PIPELINE.multi()
        PIPELINE.lpush(PROXY_YY_KEY, proxy_ip)
        PIPELINE.ltrim(PROXY_YY_KEY, 0, PROXY_NUM - 1)
        PIPELINE.execute()
    except Exception as e:
        logger.error('{tid}: {e}', tid=threading.current_thread().name, e=e)
        return False
    else:
        return True


def get_firefox_driver():
    """
    获取火狐浏览器对象
    :return:
    """
    options = webdriver.FirefoxOptions()
    options.log.level = 'fatal'
    # options.add_argument('--headless')  # 浏览器不提供可视化页面
    # options.add_argument('--start-maximized')  # 最大化运行（全屏窗口）,不设置，取元素会报错
    options.add_argument('--disable-infobars')  # 禁用浏览器正在被自动化程序控制的提示
    options.add_argument('--incognito')  # 隐身模式（无痕模式）
    driver = webdriver.Firefox(options=options, service_log_path='../logs/geckodriver.log')
    return driver


def run(driver):
    """
    执行函数
    :param driver: 浏览器对象
    :return:
    """
    while True:
        # 获取redis库中代理池长度
        key_len = get_redis_key_len()
        # 判断是否达到最大值
        if key_len < PROXY_NUM:
            try:
                # 获取代理IP
                proxy_ip = get_proxy_ip()
                try:
                    # 检查代理IP
                    assert check_proxy_ip(driver, proxy_ip)
                    # 保存代理IP
                    assert save_to_redis(proxy_ip)
                except UnexpectedAlertPresentException:
                    break
                except:
                    continue
            except:
                time.sleep(1 * 60)  # 添加延时，防止代理服务故障时访问过于频繁，使得会话连接数超过上限
                continue
        else:
            time.sleep(0.5)
            continue


def main():
    """
    主函数
    :return:
    """
    CONN.delete(PROXY_YY_KEY)
    logger.add('../logs/runtime_{time}.log', rotation='00:00', retention='7 days')
    with ThreadPoolExecutor(6, thread_name_prefix='Proxy_Thread') as tpl:
        [tpl.submit(run, get_firefox_driver()) for _ in range(5)]
        tpl.submit(delete_redis_key, 60)


if __name__ == '__main__':
    main()
